{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Extracting ./data\\train-images-idx3-ubyte.gz\n",
      "Extracting ./data\\train-labels-idx1-ubyte.gz\n",
      "Extracting ./data\\t10k-images-idx3-ubyte.gz\n",
      "Extracting ./data\\t10k-labels-idx1-ubyte.gz\n",
      "step 100, entropy loss: 0.253527, l2_loss: 35130.507812, total loss: 2.712662\n",
      "0.9295\n",
      "0.9192\n",
      "step 200, entropy loss: 0.193268, l2_loss: 33657.417969, total loss: 2.549288\n",
      "0.957\n",
      "0.9406\n",
      "step 300, entropy loss: 0.151192, l2_loss: 32253.744141, total loss: 2.408955\n",
      "0.962\n",
      "0.95\n",
      "step 400, entropy loss: 0.126784, l2_loss: 30912.824219, total loss: 2.290682\n",
      "0.9695\n",
      "0.9553\n",
      "step 500, entropy loss: 0.093299, l2_loss: 29630.400391, total loss: 2.167428\n",
      "0.978\n",
      "0.9593\n",
      "step 600, entropy loss: 0.091885, l2_loss: 28403.537109, total loss: 2.080132\n",
      "0.9835\n",
      "0.959\n",
      "step 700, entropy loss: 0.078760, l2_loss: 27230.037109, total loss: 1.984862\n",
      "0.98\n",
      "0.9627\n",
      "step 800, entropy loss: 0.048967, l2_loss: 26106.660156, total loss: 1.876433\n",
      "0.99\n",
      "0.9652\n",
      "step 900, entropy loss: 0.064574, l2_loss: 25031.509766, total loss: 1.816780\n",
      "0.9855\n",
      "0.9641\n",
      "step 1000, entropy loss: 0.043597, l2_loss: 24002.544922, total loss: 1.723775\n",
      "0.9935\n",
      "0.9663\n",
      "step 1100, entropy loss: 0.042874, l2_loss: 23017.214844, total loss: 1.654080\n",
      "0.993\n",
      "0.9688\n",
      "step 1200, entropy loss: 0.033110, l2_loss: 22074.054688, total loss: 1.578294\n",
      "0.996\n",
      "0.9682\n",
      "step 1300, entropy loss: 0.031518, l2_loss: 21170.970703, total loss: 1.513486\n",
      "0.9955\n",
      "0.9699\n",
      "step 1400, entropy loss: 0.035829, l2_loss: 20306.044922, total loss: 1.457252\n",
      "0.997\n",
      "0.9705\n",
      "step 1500, entropy loss: 0.019094, l2_loss: 19477.687500, total loss: 1.382533\n",
      "0.998\n",
      "0.9699\n",
      "step 1600, entropy loss: 0.019378, l2_loss: 18684.009766, total loss: 1.327259\n",
      "0.997\n",
      "0.9713\n",
      "step 1700, entropy loss: 0.018050, l2_loss: 17923.748047, total loss: 1.272713\n",
      "0.9985\n",
      "0.9718\n",
      "step 1800, entropy loss: 0.021622, l2_loss: 17195.443359, total loss: 1.225303\n",
      "0.997\n",
      "0.973\n",
      "step 1900, entropy loss: 0.014962, l2_loss: 16497.533203, total loss: 1.169789\n",
      "1.0\n",
      "0.972\n",
      "step 2000, entropy loss: 0.020629, l2_loss: 15828.754883, total loss: 1.128642\n",
      "0.998\n",
      "0.971\n",
      "step 2100, entropy loss: 0.014442, l2_loss: 15187.904297, total loss: 1.077596\n",
      "1.0\n",
      "0.9733\n",
      "step 2200, entropy loss: 0.011645, l2_loss: 14573.561523, total loss: 1.031794\n",
      "1.0\n",
      "0.9731\n",
      "step 2300, entropy loss: 0.010589, l2_loss: 13984.845703, total loss: 0.989528\n",
      "1.0\n",
      "0.9734\n",
      "step 2400, entropy loss: 0.011365, l2_loss: 13420.642578, total loss: 0.950810\n",
      "0.9995\n",
      "0.9736\n",
      "step 2500, entropy loss: 0.011708, l2_loss: 12879.829102, total loss: 0.913297\n",
      "0.9985\n",
      "0.9731\n",
      "step 2600, entropy loss: 0.008887, l2_loss: 12361.518555, total loss: 0.874194\n",
      "1.0\n",
      "0.9744\n",
      "step 2700, entropy loss: 0.009262, l2_loss: 11864.710938, total loss: 0.839792\n",
      "1.0\n",
      "0.9738\n",
      "step 2800, entropy loss: 0.009163, l2_loss: 11388.518555, total loss: 0.806359\n",
      "1.0\n",
      "0.9744\n",
      "step 2900, entropy loss: 0.008370, l2_loss: 10932.166992, total loss: 0.773622\n",
      "1.0\n",
      "0.9745\n",
      "step 3000, entropy loss: 0.007177, l2_loss: 10494.692383, total loss: 0.741805\n",
      "1.0\n",
      "0.9738\n",
      "step 3100, entropy loss: 0.007187, l2_loss: 10075.494141, total loss: 0.712472\n",
      "1.0\n",
      "0.9744\n",
      "step 3200, entropy loss: 0.006910, l2_loss: 9673.651367, total loss: 0.684066\n",
      "1.0\n",
      "0.9765\n",
      "step 3300, entropy loss: 0.007608, l2_loss: 9288.484375, total loss: 0.657802\n",
      "1.0\n",
      "0.9755\n",
      "step 3400, entropy loss: 0.007917, l2_loss: 8919.399414, total loss: 0.632275\n",
      "1.0\n",
      "0.9765\n",
      "step 3500, entropy loss: 0.005921, l2_loss: 8565.747070, total loss: 0.605524\n",
      "1.0\n",
      "0.9768\n",
      "step 3600, entropy loss: 0.007802, l2_loss: 8226.631836, total loss: 0.583666\n",
      "1.0\n",
      "0.9764\n",
      "step 3700, entropy loss: 0.006868, l2_loss: 7901.622559, total loss: 0.559981\n",
      "1.0\n",
      "0.9764\n",
      "step 3800, entropy loss: 0.006402, l2_loss: 7590.124023, total loss: 0.537711\n",
      "1.0\n",
      "0.9776\n",
      "step 3900, entropy loss: 0.009575, l2_loss: 7291.558105, total loss: 0.519984\n",
      "1.0\n",
      "0.9767\n",
      "step 4000, entropy loss: 0.006499, l2_loss: 7005.457520, total loss: 0.496881\n",
      "1.0\n",
      "0.9774\n",
      "step 4100, entropy loss: 0.007212, l2_loss: 6731.256836, total loss: 0.478400\n",
      "1.0\n",
      "0.977\n",
      "step 4200, entropy loss: 0.006500, l2_loss: 6468.386719, total loss: 0.459287\n",
      "1.0\n",
      "0.9773\n",
      "step 4300, entropy loss: 0.006538, l2_loss: 6216.496582, total loss: 0.441692\n",
      "1.0\n",
      "0.9775\n",
      "step 4400, entropy loss: 0.006051, l2_loss: 5975.057617, total loss: 0.424306\n",
      "1.0\n",
      "0.9779\n",
      "step 4500, entropy loss: 0.006030, l2_loss: 5743.602539, total loss: 0.408082\n",
      "1.0\n",
      "0.9774\n",
      "step 4600, entropy loss: 0.006147, l2_loss: 5630.196289, total loss: 0.400260\n",
      "1.0\n",
      "0.9784\n",
      "step 4700, entropy loss: 0.005688, l2_loss: 5520.317383, total loss: 0.392110\n",
      "1.0\n",
      "0.9782\n",
      "step 4800, entropy loss: 0.005360, l2_loss: 5412.719238, total loss: 0.384250\n",
      "1.0\n",
      "0.9784\n",
      "step 4900, entropy loss: 0.006340, l2_loss: 5307.374023, total loss: 0.377856\n",
      "1.0\n",
      "0.9781\n",
      "step 5000, entropy loss: 0.006362, l2_loss: 5204.263672, total loss: 0.370661\n",
      "1.0\n",
      "0.978\n",
      "step 5100, entropy loss: 0.005071, l2_loss: 5103.333984, total loss: 0.362304\n",
      "1.0\n",
      "0.9784\n",
      "step 5200, entropy loss: 0.006474, l2_loss: 5004.478516, total loss: 0.356788\n",
      "0.9995\n",
      "0.9787\n",
      "step 5300, entropy loss: 0.006500, l2_loss: 4907.735840, total loss: 0.350041\n",
      "1.0\n",
      "0.9786\n",
      "step 5400, entropy loss: 0.005485, l2_loss: 4813.036621, total loss: 0.342398\n",
      "1.0\n",
      "0.9782\n",
      "step 5500, entropy loss: 0.005985, l2_loss: 4720.289062, total loss: 0.336405\n",
      "1.0\n",
      "0.9793\n",
      "step 5600, entropy loss: 0.005748, l2_loss: 4629.509277, total loss: 0.329814\n",
      "1.0\n",
      "0.9787\n",
      "step 5700, entropy loss: 0.006205, l2_loss: 4540.630859, total loss: 0.324050\n",
      "1.0\n",
      "0.9792\n",
      "step 5800, entropy loss: 0.006005, l2_loss: 4453.635254, total loss: 0.317759\n",
      "1.0\n",
      "0.9789\n",
      "step 5900, entropy loss: 0.005882, l2_loss: 4368.436035, total loss: 0.311672\n",
      "1.0\n",
      "0.9793\n",
      "step 6000, entropy loss: 0.007059, l2_loss: 4285.005859, total loss: 0.307010\n",
      "1.0\n",
      "0.9797\n",
      "step 6100, entropy loss: 0.006439, l2_loss: 4267.871094, total loss: 0.305190\n",
      "1.0\n",
      "0.9792\n",
      "step 6200, entropy loss: 0.006260, l2_loss: 4251.457031, total loss: 0.303862\n",
      "1.0\n",
      "0.9795\n",
      "step 6300, entropy loss: 0.005996, l2_loss: 4235.103516, total loss: 0.302454\n",
      "1.0\n",
      "0.9797\n",
      "step 6400, entropy loss: 0.005371, l2_loss: 4218.817383, total loss: 0.300689\n",
      "1.0\n",
      "0.9797\n",
      "step 6500, entropy loss: 0.006642, l2_loss: 4202.599609, total loss: 0.300824\n",
      "0.9995\n",
      "0.9798\n",
      "step 6600, entropy loss: 0.006086, l2_loss: 4186.457520, total loss: 0.299138\n",
      "1.0\n",
      "0.9797\n",
      "step 6700, entropy loss: 0.005804, l2_loss: 4170.381836, total loss: 0.297731\n",
      "1.0\n",
      "0.9795\n",
      "step 6800, entropy loss: 0.006258, l2_loss: 4154.367188, total loss: 0.297064\n",
      "1.0\n",
      "0.9797\n",
      "step 6900, entropy loss: 0.005745, l2_loss: 4138.430664, total loss: 0.295435\n",
      "1.0\n",
      "0.9796\n",
      "step 7000, entropy loss: 0.005418, l2_loss: 4122.555176, total loss: 0.293997\n",
      "1.0\n",
      "0.9797\n",
      "step 7100, entropy loss: 0.005894, l2_loss: 4114.564453, total loss: 0.293913\n",
      "1.0\n",
      "0.9796\n",
      "step 7200, entropy loss: 0.006662, l2_loss: 4106.667969, total loss: 0.294128\n",
      "1.0\n",
      "0.9798\n",
      "step 7300, entropy loss: 0.006236, l2_loss: 4098.787598, total loss: 0.293151\n",
      "1.0\n",
      "0.9799\n",
      "step 7400, entropy loss: 0.005468, l2_loss: 4090.926758, total loss: 0.291833\n",
      "1.0\n",
      "0.9799\n",
      "step 7500, entropy loss: 0.006295, l2_loss: 4083.077881, total loss: 0.292110\n",
      "1.0\n",
      "0.9798\n",
      "step 7600, entropy loss: 0.006234, l2_loss: 4075.247314, total loss: 0.291501\n",
      "1.0\n",
      "0.9799\n",
      "step 7700, entropy loss: 0.005543, l2_loss: 4067.435303, total loss: 0.290263\n",
      "1.0\n",
      "0.9799\n",
      "step 7800, entropy loss: 0.006243, l2_loss: 4059.640625, total loss: 0.290418\n",
      "1.0\n",
      "0.9797\n",
      "step 7900, entropy loss: 0.004998, l2_loss: 4051.860840, total loss: 0.288628\n",
      "1.0\n",
      "0.9801\n",
      "step 8000, entropy loss: 0.005037, l2_loss: 4044.096436, total loss: 0.288124\n",
      "1.0\n",
      "0.9798\n",
      "step 8100, entropy loss: 0.005003, l2_loss: 4042.483887, total loss: 0.287977\n",
      "1.0\n",
      "0.9799\n",
      "step 8200, entropy loss: 0.006555, l2_loss: 4040.932373, total loss: 0.289420\n",
      "0.9995\n",
      "0.98\n",
      "step 8300, entropy loss: 0.005921, l2_loss: 4039.384277, total loss: 0.288678\n",
      "1.0\n",
      "0.9799\n",
      "step 8400, entropy loss: 0.005917, l2_loss: 4037.835449, total loss: 0.288565\n",
      "1.0\n",
      "0.98\n",
      "step 8500, entropy loss: 0.005469, l2_loss: 4036.285889, total loss: 0.288009\n",
      "1.0\n",
      "0.98\n",
      "step 8600, entropy loss: 0.005989, l2_loss: 4034.739746, total loss: 0.288420\n",
      "1.0\n",
      "0.98\n",
      "step 8700, entropy loss: 0.005530, l2_loss: 4033.191162, total loss: 0.287854\n",
      "1.0\n",
      "0.9799\n",
      "step 8800, entropy loss: 0.005271, l2_loss: 4031.645264, total loss: 0.287486\n",
      "1.0\n",
      "0.98\n",
      "step 8900, entropy loss: 0.006492, l2_loss: 4030.099854, total loss: 0.288599\n",
      "1.0\n",
      "0.98\n",
      "step 9000, entropy loss: 0.005118, l2_loss: 4028.553223, total loss: 0.287116\n",
      "1.0\n",
      "0.9799\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "step 9100, entropy loss: 0.006295, l2_loss: 4027.010254, total loss: 0.288186\n",
      "1.0\n",
      "0.9799\n",
      "step 9200, entropy loss: 0.005599, l2_loss: 4025.465088, total loss: 0.287382\n",
      "1.0\n",
      "0.98\n",
      "step 9300, entropy loss: 0.004844, l2_loss: 4023.921631, total loss: 0.286519\n",
      "1.0\n",
      "0.9801\n",
      "step 9400, entropy loss: 0.005946, l2_loss: 4022.380859, total loss: 0.287512\n",
      "1.0\n",
      "0.98\n",
      "step 9500, entropy loss: 0.005709, l2_loss: 4020.838379, total loss: 0.287168\n",
      "1.0\n",
      "0.98\n",
      "step 9600, entropy loss: 0.005063, l2_loss: 4019.296631, total loss: 0.286414\n",
      "1.0\n",
      "0.9801\n",
      "step 9700, entropy loss: 0.005398, l2_loss: 4017.756104, total loss: 0.286641\n",
      "1.0\n",
      "0.98\n",
      "step 9800, entropy loss: 0.005549, l2_loss: 4016.218506, total loss: 0.286685\n",
      "1.0\n",
      "0.9801\n",
      "step 9900, entropy loss: 0.005634, l2_loss: 4014.678711, total loss: 0.286662\n",
      "1.0\n",
      "0.9801\n",
      "step 10000, entropy loss: 0.004737, l2_loss: 4013.139404, total loss: 0.285657\n",
      "1.0\n",
      "0.98\n"
     ]
    }
   ],
   "source": [
    "\"\"\"A very simple MNIST classifier.\n",
    "See extensive documentation at\n",
    "https://www.tensorflow.org/get_started/mnist/beginners\n",
    "\"\"\"\n",
    "from __future__ import absolute_import\n",
    "from __future__ import division\n",
    "from __future__ import print_function\n",
    "\n",
    "import argparse\n",
    "import sys\n",
    "\n",
    "from tensorflow.examples.tutorials.mnist import input_data\n",
    "\n",
    "import tensorflow as tf\n",
    "\n",
    "FLAGS = None\n",
    "\n",
    "\n",
    "#我们在这里调用系统提供的Mnist数据函数为我们读入数据，如果没有下载的话则进行下载。\n",
    "\n",
    "#<font color=#ff0000>**这里将data_dir改为适合你的运行环境的目录**</font>\n",
    "\n",
    "# Import data\n",
    "data_dir = './data'\n",
    "mnist = input_data.read_data_sets(data_dir, one_hot=True)\n",
    "\n",
    "def initialize(shape, stddev=0.1):\n",
    "  return tf.truncated_normal(shape, stddev=0.1)\n",
    "\n",
    "def swish(x):\n",
    "  return x*tf.nn.sigmoid(x)\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "def activation(x):\n",
    "#  return selu(x)\n",
    "#  return relu(x)\n",
    "#  return sigmoid(x)\n",
    "  return swish(x)\n",
    "\n",
    "def add_layer(inputs,in_size,out_size,stddev=0.1,activation_function=None):\n",
    "    Weights=tf.Variable(tf.random_normal([in_size,out_size],stddev=stddev))\n",
    "  \n",
    "    biases=tf.Variable(tf.random_normal([out_size],stddev=0.1)) \n",
    "    Wx_plus_b=tf.matmul(inputs,Weights)+biases\n",
    "    if activation_function is None:\n",
    "        outputs=Wx_plus_b\n",
    "    else:\n",
    "        outputs=activation_function(Wx_plus_b)\n",
    "    return outputs,Wx_plus_b,Weights\n",
    "\n",
    "#一个非常非常简陋的模型\n",
    "\n",
    "# Create the model\n",
    "'''\n",
    "L1_units_count = 100\n",
    "x = tf.placeholder(tf.float32, [None, 784])\n",
    "W1 = tf.Variable(initialize([784, L1_units_count], stddev=0.05))\n",
    "b_1 = tf.Variable(initialize([L1_units_count]))\n",
    "logits_1 = tf.matmul(x, W1) + b_1\n",
    "output_1 = activation(logits_1)\n",
    "\n",
    "L2_units_count = 10 \n",
    "W2 = tf.Variable(initialize([L1_units_count, L2_units_count], stddev=0.063))\n",
    "b_2 = tf.Variable(initialize([L2_units_count]))\n",
    "logits_2 = tf.matmul(output_1, W2) + b_2  \n",
    "\n",
    "y = logits_2\n",
    "'''\n",
    "\n",
    "learningrate = tf.placeholder(tf.float32 )\n",
    "\n",
    "x = tf.placeholder(tf.float32, [None, 784])\n",
    "\n",
    "L_1,Wpb1,W1=add_layer(x,784,300,stddev=0.53,activation_function=swish)\n",
    "'''\n",
    "L_2,W2=add_layer(L_1,500,300,stddev=0.063,activation_function=swish)\n",
    "\n",
    "L_3,W3=add_layer(L_2,300,200,stddev=0.043,activation_function=swish)\n",
    "L_4,W4=add_layer(L_3,200,150,stddev=0.033,activation_function=swish)\n",
    "\n",
    "L_5,W5=add_layer(L_4,150,100,stddev=0.073,activation_function=swish)\n",
    "\n",
    "L_6,W6=add_layer(L_5,200,100,stddev=0.083,activation_function=swish)\n",
    "\n",
    "L_2,W2=add_layer(L_1,500,100,stddev=0.063,activation_function=swish)\n",
    "'''\n",
    "#L_2,Wpb2,W2=add_layer(L_1,500,100,stddev=0.069,activation_function=swish)\n",
    "\n",
    "\n",
    "L_3,Wpb3,W3=add_layer(L_1,300,10,stddev=0.062,activation_function=swish)\n",
    "y=Wpb3\n",
    "\n",
    "l2_loss = tf.nn.l2_loss(W1)+ tf.nn.l2_loss(W3)\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "#定义我们的ground truth 占位符\n",
    "\n",
    "# Define loss and optimizer\n",
    "y_ = tf.placeholder(tf.float32, [None, 10])\n",
    "\n",
    "#接下来我们计算交叉熵，注意这里不要使用注释中的手动计算方式，而是使用系统函数。\n",
    "#另一个注意点就是，softmax_cross_entropy_with_logits的logits参数是**未经激活的wx+b**\n",
    "\n",
    "# The raw formulation of cross-entropy,\n",
    "#\n",
    "#   tf.reduce_mean(-tf.reduce_sum(y_ * tf.log(tf.nn.softmax(y)),\n",
    "#                                 reduction_indices=[1]))\n",
    "#\n",
    "# can be numerically unstable.\n",
    "#\n",
    "# So here we use tf.nn.softmax_cross_entropy_with_logits on the raw\n",
    "# outputs of 'y', and then average across the batch.\n",
    "cross_entropy = tf.reduce_mean(\n",
    "  #  tf.nn.softmax_cross_entropy_with_logits(labels=y_, logits=y))\n",
    "     tf.nn.softmax_cross_entropy_with_logits(labels=y_, logits=y))\n",
    "#l2_loss = tf.nn.l2_loss(W1) + tf.nn.l2_loss(W2)\n",
    "total_loss = cross_entropy + 7e-5*l2_loss\n",
    "\n",
    "#生成一个训练step\n",
    "\n",
    "#train_step = tf.train.GradientDescentOptimizer(learningrate).minimize(total_loss )\n",
    "#train_step = tf.train.GradientDescentOptimizer(0.5).minimize(cross_entropy)\n",
    "#train_step = tf.train.AdadeltaOptimizer(0.5).minimize(cross_entropy)\n",
    "#train_step = tf.train.AdamOptimizer(learningrate).minimize(cross_entropy)\n",
    "train_step = tf.train.AdagradOptimizer(learningrate).minimize(total_loss)\n",
    "#train_step = tf.train.FtrlOptimizer(0.5).minimize(cross_entropy)\n",
    "#train_step = tf.train.MomentumOptimizer(0.5).minimize(cross_entropy)\n",
    "#train_step = tf.train.ProximalAdagradOptimizer(0.5).minimize(cross_entropy)\n",
    "#train_step = tf.train.RMSPropOptimizer(0.5).minimize(cross_entropy)\n",
    "#train_step = tf.train.SyncReplicasOptimizer(0.5).minimize(cross_entropy)\n",
    "\n",
    "sess = tf.Session()\n",
    "init_op = tf.global_variables_initializer()\n",
    "sess.run(init_op)\n",
    "\n",
    "#在这里我们仍然调用系统提供的读取数据，为我们取得一个batch。\n",
    "#然后我们运行3k个step(5 epochs)，对权重进行优化。\n",
    "\n",
    "# Train\n",
    "correct_prediction = tf.equal(tf.argmax(y, 1), tf.argmax(y_, 1))\n",
    "accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))\n",
    "lr=0.5\n",
    "for step in range(10000):\n",
    "    \n",
    "   # batch_xs, batch_ys = mnist.train.next_batch(100)\n",
    "   # sess.run(train_step, feed_dict={x: batch_xs, y_: batch_ys,learningrate:lr})\n",
    "   # print(sess.run(accuracy, feed_dict={x: batch_xs, y_: batch_ys}))\n",
    "   # print(sess.run(accuracy, feed_dict={x: mnist.test.images,\n",
    "   #                                y_: mnist.test.labels}))\n",
    "    \n",
    "    batch_xs, batch_ys = mnist.train.next_batch(2000)\n",
    "    if step<4500:\n",
    "        lr = 1.0\n",
    "    elif step<6000:\n",
    "        lr = 0.5\n",
    "    elif step<7000:\n",
    "        lr = 0.1\n",
    "    elif step<8000:\n",
    "        lr = 0.05\n",
    "    elif step<10000:\n",
    "        lr = 0.01\n",
    "    \n",
    "    else:\n",
    "        lr = 0.0005\n",
    "    _,cross_entropyv,l2_lossv, total_lossv=sess.run([train_step, cross_entropy, l2_loss, total_loss],  feed_dict={x: batch_xs, y_: batch_ys,learningrate:lr})\n",
    "    if (step+1) % 100 == 0:  \n",
    "      # \n",
    "       # _,cross_entropyv=sess.run([train_step, cross_entropy],  feed_dict={x: batch_xs, y_: batch_ys,learningrate:lr})\n",
    "        print('step %d, entropy loss: %f, l2_loss: %f, total loss: %f' % (step+1, cross_entropyv, l2_lossv, total_lossv))\n",
    "        #print('step %d, entropy loss: %f' % (step+1, cross_entropyv))\n",
    "        print(sess.run(accuracy, feed_dict={x: batch_xs, y_: batch_ys}))\n",
    "        print(sess.run(accuracy, feed_dict={x: mnist.test.images,y_: mnist.test.labels}))\n",
    "                                    \n",
    "     \n",
    "   \n",
    "\n",
    "    \n",
    "\n",
    "#验证我们模型在测试数据上的准确率\n",
    "\n",
    "  # Test trained model\n",
    "#correct_prediction = tf.equal(tf.argmax(y, 1), tf.argmax(y_, 1))\n",
    "#accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))\n",
    "#print(sess.run(accuracy, feed_dict={x: mnist.test.images,y_: mnist.test.labels}))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "毫无疑问，这个模型是一个非常简陋，性能也不理想的模型。目前只能达到92%左右的准确率。\n",
    "接下来，希望大家利用现有的知识，将这个模型优化至98%以上的准确率。\n",
    "Hint：\n",
    "- 多隐层\n",
    "- 激活函数\n",
    "- 正则化\n",
    "- 初始化\n",
    "- 摸索一下各个超参数\n",
    "  - 隐层神经元数量\n",
    "  - 学习率\n",
    "  - 正则化惩罚因子\n",
    "  - 最好每隔几个step就对loss、accuracy等等进行一次输出，这样才能有根据地进行调整"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
